'''
Created on 2010-08-29

'''

import sys
import os.path
import re
import httplib
from lcboItem import lcboItem
from lcboItemPickler import itemPickler
from sqloader import sqlloader

class lcboParse(object):
    '''
    Parses the lcbo's website grabbing information about each item available
    for sale.
    
    '''
        
    def __init__(self, path, pickle=False):
        self.url = '''lcbo.com''' #: url of the lcbo's webiste
        
        self.searchLink = '''/lcbo-ear/lcbo/product/searchResults.do'''
        '''This is the path where items are displayed. In order to progress
        to the next page and list the next x items a post command must be
        issued. The items on this page are parsed for relevant information.'''
        
        self.productLink = '''/lcbo-ear/lcbo/product'''\
        +'''/details.do?language=EN&itemNumber='''
        '''Not used??'''
        
        self.printerLink = '''/lcbo-ear/lcbo/product/'''\
        +'''printerFriendly.do?language=EN&itemNumber='''
        ''' This is the link for a product's printer friendly information page.
        The products information is parsed from this page.'''
        
        self.post = '''STOCK_TYPE_NAME=&ITEM_NAME=&KEYWORDS='''+\
        '''&ITEM_NUMBER=&productListingType=&LIQUOR_TYPE_SHORT_=*'''+\
        '''&CATEGORY_NAME=*&SUB_CATEGORY_NAME=*&PRODUCING_CNAME=*'''+\
        '''&PRODUCING_REGION_N=*&UNIT_VOLUME=*&SELLING_PRICE=*'''+\
        '''&LTO_SALES_CODE=N&VQA_CODE=N&KOSHER_CODE=N&VINTAGES_CODE=N'''+\
        '''&VALUE_ADD_SALES_CO=N&AIR_MILES_SALES_CO=N&language=EN'''+\
        '''&style=+LCBO.css&sort=sortedProduct&order=1&resultsPerPage=100'''+\
        '''&page=1&action=result&sortby=sortedProduct&orderby=&numPerPage=100'''
        '''The post command issued so we can access the next 100 items off the
        search link defined above'''
        
        self.connection = None #: the active connection if any
   
        self.debug = True #:debug mode
    
        self.itemObjs = [] #:a collection of the parsed items.
        
        #Do we want to pickle the results?
        if pickle:
            self.pickle = True
            self.pickler = itemPickler(path)
            '''An lcboItemPickler. Used instead of writing the results to an
            sqlite3 database'''
        else:
            self.pickle = False
            self.sql = sqlloader(path) 
            '''An sqloader object. Used to write the results to an sql3 file'''
            self.sql.setupLcboDatabase()
                        
    def getItems(self):
        '''Parse the items found on the lcbo website'''
        
        reply = self.openUrl(self.searchLink,"POST",self.post)
        
        if reply is None:
            print '''Couldn't open main search window, exiting'''
            if self.debug: print self.errors
            self.cleanUp()
        
        '''Regular expression searching for how many items are in the lcbo's
        database.'''
        numberRX = re.compile(r'''Results:\s*<b>\d*\s*-''' + \
                              r'''(?P<pageNum>\s*\d*)\s*of\s*(?P<total>\d*)''')
        result = numberRX.search(reply)
        
        if result is None:
            print 'No search results found (peculiar)'
            self.cleanup()
        
        totalItems = int(result.group('total'))
        resultsPerPage = 100
        totalPages = totalItems / resultsPerPage
        
        if totalItems % resultsPerPage > 0: totalPages = totalPages + 1
        
        #process the first page out of the loop since we already just grabbed it
        print 'parsing page no: ' + '1' + ' of ' + `totalPages` + ' %' +'%.2f'\
                                                % (1.0/totalPages*100)
        items = self.parseItems(reply)
        self.parseProduct(items)
        
        for page in range(2,totalPages+1):
            print 'parsing page no: ' + `page` + ' of ' + `totalPages`\
                                    + ' ' + items[-1] + ' %' + '%.2f' \
                                    % ((1.0*page)/totalPages*100)
            contentsOf = self.nextPage(page)
            if contentsOf is None:
                if self.debug:
                    self.logDebug("Could not load page number (" + page + ")")
                continue
            items = self.parseItems(contentsOf)
            self.parseProduct(items)
            
        #did the program log any errors?    
        if getattr(self,'errors',None) is not None: print self.errors
        
        #Do we want to pickle the items for later use?
        if self.pickle:
            self.dumpItems()
        else:
            self.writeToSQL()
                            
        self.cleanUp()
            
    def nextPage(self,pageNo):
        '''Issue a post for the next page
        @param pageNo: The page number signifies which page of results 
        out of all total possible results we are currently looking at.
        @type pageNo: int
        
        @return: the html contents of the current page of results we are on
        (pageNo being the page)
        @rtype: string
        
        '''
        
        self.post = self.post.replace('page='+`(pageNo-1)`,'page='+`pageNo`)
        response = self.openUrl(self.searchLink,"POST",self.post)
        return response
        
    def parseItems(self,contents):
        '''Parse the item id numbers from the search result page and
        return them in a list
        
        @param contents: An html page listing lcbo items. This page is the 
        result of issuing a post command on the search link self.searchLink
        @type contents: string
        
        @return: a list of parsed item id numbers.
        @rtype: string list
        
        ''' 
        items = []
        itemRegex = re.compile(r'''itemNumber=(?P<item>\d+)''')
        
        for item in itemRegex.finditer(contents):
            itemId = item.group('item')
            #print itemId
            if itemId not in items: #why the duplicate check?
                items.append(itemId)
        return items
        
    def parseProduct(self,items):
        '''Parse information about a single lcbo item.
        Create a new lcbo item from the information parsed from an item's
        'printer friendly' link.
        
        @param items: a list of id numbers where when an individual id number
        is combined with the self.printerLink it yields us the printer friendly 
        for that item.
        @type items: string list
          
        '''
        
        for item in items:
            details = self.openUrl(self.printerLink+item)
            
            if details is None: continue #broken URL
            
            alcohol = re.compile('''Alcohol/Vol.<BR>''')
            if alcohol.search(details) == None: 
                continue #non alcoholic sales item
            
            title = re.compile(r'''"medium-title">(?P<title>.*?)<''')
            quantity =  re.compile(r'''<br>.*?\|(?P<item>.*?)<br>''',re.DOTALL)
            category = re.compile(r'''(?P<categories>(Beer|Spirits|Wine)'''\
                                    +r'''.*?)<br>''')
            price = re.compile(r'''Price:\s*\$\s*(?P<price>\d+\.?\d+)''')
            percent = re.compile(r'''(?P<percent>.*?)%\s*Alcohol/Vol.<BR>''')
            geography = re.compile(r'''Made in:(?P<location>.*?)<BR>''')
            brewer = re.compile(r'''By:(?P<brewer>.*?)<BR>''')
            
            pTitle = self.parseField(title,details,'title',item)
            pQuantity = self.parseField(quantity,details,'item',item)
            pCategory = self.parseField(category,details,'categories',item)
            pPrice = (self.parseField(price,details,'price',item))
            pPercent = (self.parseField(percent,details,'percent',item))
            pGeography = self.parseField(geography,details,'location',item)
            pBrewer = self.parseField(brewer,details,'brewer',item)
            
            #If we are logging errors, well... log the error.
            if self.debug:
                if pTitle is None: self.logDebug('Title for ' + item \
                                                 + ' is missing')
                if pQuantity is None: self.logDebug('Quantity for ' + item \
                                                    + ' is missing')
                if pPrice is None: self.logDebug('Price for ' + item \
                                                 + ' is missing')
                if pCategory is None: self.logDebug('Category for ' + item \
                                                    + ' is missing')
                if pPercent is None: self.logDebug('Percent for ' + item \
                                                   + ' is missing')
                if pGeography is None: self.logDebug('Title for ' + item \
                                                     + ' is missing')
                if pBrewer is None: self.logDebug('Title for ' + item \
                                                  + ' is missing')
            
            
            self.addItem(pTitle,pQuantity,pPrice,pCategory,
                             pPercent,pGeography,pBrewer)
                
    def parseField(self,regex,contents,group,item):
        ''' Parse information for a single field.
        Use the passed in regular expression to parse information for a 
        particular field from an item's 'printer friendly' page. 
        
        @param regex: A compiled regular expression for a particular field.
        @type regex: compiled regular expression
        
        @param contents: An html page with info about a single item which
        will be parsed by the regular expression.
        @type contents: string
        
        @param group: A regular expression group name, usually related in 
        some way to the regular expression. i.e 'title' gives us parsed 
        title information when the regex is a compiled title regex. 
        @type group: string
        
        @param item: Id number of the item we are parsing. Used for debugging.
        @type item: string
        
        '''
        
        pField = regex.search(contents)
        
        if pField is None:
            if self.debug: 
                self.logDebug(item + " has no " + group)
        else:
            pField = pField.group(group)
        return pField
    
    def addItem(self,pTitle,pQuantity,pPrice,pCategory,pPercent,pGeography,pBrewer):
        '''Add an item to the current collection of lcbo items'''
        
        lcItem = lcboItem(pTitle
                          ,pQuantity
                          ,pPrice
                          ,pCategory
                          ,pPercent
                          ,pGeography
                          ,pBrewer)
        self.itemObjs.append(lcItem)
        
    def dumpItems(self):
        '''Pickle the lcbo items'''
        for item in self.itemObjs:
            self.pickler.pickleItem(item)
        self.pickler.close()
    
    def writeToSQL(self):
        '''Write the items to an sqlite3 database'''
        for item in self.itemObjs:
            self.sql.addLcboItem(item)
        self.sql.close()
        
    def logDebug(self,error):
        '''Log an error message'''
        if getattr(self,'errors',None) is None:
            self.errors = error
        else:
            self.errors += '\n' + error
        
    def openUrl(self,link,method="GET",post=None):
        '''Open and return the contents of a URL.
        
        @param link: The url to be opened.
        @type link: string
        
        @param method: The method of retrieval, post or get.
        @type method: string
        
        @param post: post data used iff the method is a get.
        @type post: string of key value pairs. i.e STOCK=GREAT&MORE=BETTER
        
        @return: the html contents of the opened URL.
        @rtype: string
        
        '''
        
        if self.connection == None:
            self.connection = httplib.HTTPConnection(self.url)
            self.headers = {'Content-Type' : 'application/x-www-form-urlencoded'}
            
        con = self.connection
        if(method is "GET"):
            con.request("GET",link)
        else:
            con.request("POST",link,post,self.headers)
        response = con.getresponse()

        status = response.status
        if status != 200:
            if self.debug:
                self.logDebug("""Could not retrieve Url:""" + link)
            return None
        return response.read()
    
    def cleanUp(self):
        '''Closes the connection to the server if it is currently active'''
        if self.connection is not None:
            self.connection.close()
        exit()
        
def parse(pickle):
    '''Parse the command line arguments and start parsing!'''
    #if pickle is True then we are pickling the objects instead of creating an
    #sql database. 
    path = sys.argv[2]
    parsePath(path)
    lp = lcboParse(path,pickle)
    lp.getItems()
    
def parsePath(filepath):
    '''Checks that the provided location to save the parsed data is valid'''
    filepath = filepath.strip()
    if filepath == '':
        print 'Usage: beerparse database_save_location'
        exit()
        
    dirname = os.path.dirname(filepath)
        
    if dirname != '' and not os.path.exists(dirname):
        print 'invalid path specified'
        exit()
    
    if os.path.isdir(filepath):
        print 'path pointed to a directory, invalid path.'
        exit()
    
    if os.path.isfile(filepath):
        if not os.access(filepath, os.W_OK):
            print 'cannot write to path (insufficient permissions)'
            exit()
        else:
            return
            
    if dirname == '': dirname = '.'
    if not os.access(dirname,os.W_OK):
        print 'insufficient permissions to create file in supplied path' +\
        ' ' + dirname
        exit()
            
if __name__ == "__main__":
    usage = 'usage [-pickle path] [-sql path]'
    if len(sys.argv) == 2: #one arg (maybe just the path given)
        print usage
        exit()
    elif len(sys.argv) > 2: #two or more args (pickle path)
        if sys.argv[1] == '-pickle':
            parse(True)
        elif sys.argv[1] == '-sql':
            parse(False)
        else:
            print usage
    else:
        print usage
    